import re
import requests
import json
from multiprocessing import Pool

def get_one_page(url):
    header = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0'}
    response = requests.get(url, headers=header)
    if response.status_code == 200:
        return response.text
    return None

def parse_one_page(html):
    pattern = re.compile(r'<li.*?pic.*?<em.*?>(\d+)</em>.*?<a.*?href="(.*?)".*?alt="(.*?)".*?</a>.*?bd.*?<p.*?>(.*?)<br>(.*?)</p>.*?average">(.*?)</span>.*?property.*?<span>(.*?)</span>.*?inq">(.*?)</span>', re.S)
    results = re.findall(pattern, html)
    for result in results:
        content = get_one_page(result[1])
        pattern2 = re.compile(r'<span.*?summary.*?>(.*?)</span>', re.S)
        more = re.findall(pattern2, content)
        more = more[0].strip()
        more = re.sub('<br />', '', more)
        #more = more.replace(' ', '');
        more = re.sub(r'\s', '', more)
        tmp = result[3].strip()
        tmp = re.sub(r'&nbsp;', '', tmp)
        tmp = re.sub(r'/', '', tmp)
        num = tmp.find("主演")
        director = tmp[0:num]
        actor = tmp[num:]
        time = result[4].strip()[:4]
        nums = result[6].rstrip('人评价')
        yield {
            'index': result[0],
            'url': result[1],
            'title': result[2],
            'director': director[3:],
            'actor': actor[3:],
            'time': time,
            'nums': nums,
            'sub': result[7],
            'more': more
        }

def write_in_file(content):
    with open('douban_movie.json', 'a', encoding='utf8') as f:
        f.write(json.dumps(content, ensure_ascii=False)+',\n')
        f.close()

def main(offset):
    url = 'https://movie.douban.com/top250?start='+str(offset)+'&filter='
    html = get_one_page(url)
    result = parse_one_page(html)
    for res in result:
        print(res)
        write_in_file(res)

if __name__ == "__main__":
    pool = Pool()
    pool.map(main, [i*25 for i in range(10)])

